{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-5.5732729e-04,  1.4017692e+00, -5.6461025e-02, -4.0671340e-01,\n",
       "        6.5254071e-04,  1.2789289e-02,  0.0000000e+00,  0.0000000e+00],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('LunarLander-v2',\n",
    "                       continuous=True,\n",
    "                       render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, _, info = self.env.step(action)\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 400:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAZBUlEQVR4nO3de1BU9/038PfZK9ddbrIrl40+DV6QaAwqrqbTSUS3Sq2JdsZmfCLNJKYq5onVsVOmFSedacjl9yRporUZOzVxOokdnVqfscSOxaiPDVFBiBeUpDEJ2B+7eIFdUFgu+3n+oJwn6xUQ+YK+XzOfEc73u+d8zpF9e/acXdREREBEpIBBdQNEdP9iABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTLKAmjTpk0YNWoUIiIikJOTg6NHj6pqhYgUURJAf/7zn7FmzRps2LABx48fx6RJk+DxeNDQ0KCiHSJSRFPxYdScnBxMnToVGzduBACEQiGkp6fjhRdewC9+8YvBboeIFDEN9gbb29tRUVGBwsJCfZnBYEBubi7Kyspu+JhgMIhgMKh/HwqFcPnyZSQmJkLTtLveMxH1jYigubkZKSkpMBhu/kJr0APo4sWL6OrqgsPhCFvucDhw9uzZGz6muLgYL7300mC0R0QDqK6uDmlpaTcdH/QA6o/CwkKsWbNG/97v98PlcinsiHoj0hqHBbn/hWTbePyfw2uQaPsOZk5aiSjLiF6vo7PrKnYd/l8IdXTCM+PXaGtvws5/rERLy4W72DkNlNjY2FuOD3oAJSUlwWg0wufzhS33+XxwOp03fIzVaoXVah2M9mgAjfkfj2OUYybO1O7B+foqJMePh9USgwjLrX8ov62jywijwYTzvkpc8J/BmFQPssbMx5HK9yASuovd00C43SWSQb8LZrFYkJ2djdLSUn1ZKBRCaWkp3G73YLdDd0lkRBwmj/sxOjqvovz0n9DZGQRE0CUd6Aq1X1edN6muUAcAINjRgrNf7sPVjkvIHvM/ER/HM+B7gZKXYGvWrEF+fj6mTJmCadOm4a233sKVK1fwzDPPqGiH7oKx35mFkfETUfXln1HvOw0AuNj0JSpqtsGgmXu9HpEuNAX+DQA4V3cYF8bV4IHkGXgkczH2l/1vhEKdd6V/GhxKAmjx4sW4cOECioqK4PV68fDDD2Pv3r3XXZim4clktCLdOQWXrvwLx6s/1EPiXN1hnKs73O/1tgYb8flX+xFnS0NMdDIslmi0tfkHqm1SQMn7gO5UIBCA3W5X3Qbdgi16JFIcD6Hmq1KIdA3YeqMjkjAqdTrOnf+/aA0yfIY6v98Pm81203EGEBHdNbcLIH4YlYiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJSps8BdOjQIcyfPx8pKSnQNA1//etfw8ZFBEVFRRg5ciQiIyORm5uLL774ImzO5cuXsWTJEthsNsTFxeHZZ59FS0vLHe0IEQ0/fQ6gK1euYNKkSdi0adMNx1977TW8/fbb+P3vf48jR44gOjoaHo8HbW1t+pwlS5bg9OnT2LdvH/bs2YNDhw7h+eef7/9eENHwJHcAgOzatUv/PhQKidPplNdff11f1tTUJFarVT788EMREamurhYAcuzYMX3ORx99JJqmyb///e9ebdfv9wsAFos1xMvv99/yuTyg14C++uoreL1e5Obm6svsdjtycnJQVlYGACgrK0NcXBymTJmiz8nNzYXBYMCRI0duuN5gMIhAIBBWRDT8DWgAeb1eAIDD4Qhb7nA49DGv14vk5OSwcZPJhISEBH3OtYqLi2G32/VKT08fyLaJSJFhcRessLAQfr9fr7q6OtUtEdEAGNAAcjqdAACfzxe23Ofz6WNOpxMNDQ1h452dnbh8+bI+51pWqxU2my2siGj4G9AAGj16NJxOJ0pLS/VlgUAAR44cgdvtBgC43W40NTWhoqJCn7N//36EQiHk5OQMZDtENNT14aaXiIg0NzdLZWWlVFZWCgB54403pLKyUr755hsREXnllVckLi5Odu/eLSdOnJAFCxbI6NGjpbW1VV/H97//fZk8ebIcOXJEDh8+LBkZGfLUU0/1ugfeBWOxhkfd7i5YnwPo448/vuGG8vPzRaT7Vvz69evF4XCI1WqVWbNmSU1NTdg6Ll26JE899ZTExMSIzWaTZ555RpqbmxlALNY9VrcLIE1EBMNMIBCA3W5X3QYR3Ybf77/lNdthcReMiO5NDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISJk+BVBxcTGmTp2K2NhYJCcn44knnkBNTU3YnLa2NhQUFCAxMRExMTFYtGgRfD5f2Jza2lrk5eUhKioKycnJWLduHTo7O+98b4hoeJE+8Hg8snXrVjl16pRUVVXJvHnzxOVySUtLiz5n+fLlkp6eLqWlpVJeXi7Tp0+XGTNm6OOdnZ2SlZUlubm5UllZKSUlJZKUlCSFhYW97sPv9wsAFos1xMvv99/yudynALpWQ0ODAJCDBw+KiEhTU5OYzWbZsWOHPufMmTMCQMrKykREpKSkRAwGg3i9Xn3O5s2bxWazSTAY7NV2GUAs1vCo2wXQHV0D8vv9AICEhAQAQEVFBTo6OpCbm6vPGTduHFwuF8rKygAAZWVleOihh+BwOPQ5Ho8HgUAAp0+fvuF2gsEgAoFAWBHR8NfvAAqFQli9ejVmzpyJrKwsAIDX64XFYkFcXFzYXIfDAa/Xq8/5dvj0jPeM3UhxcTHsdrte6enp/W2biIaQfgdQQUEBTp06he3btw9kPzdUWFgIv9+vV11d3V3fJhHdfab+PGjVqlXYs2cPDh06hLS0NH250+lEe3s7mpqaws6CfD4fnE6nPufo0aNh6+u5S9Yz51pWqxVWq7U/rRLRUNaXi86hUEgKCgokJSVFPv/88+vGey5C79y5U1929uxZAa6/CO3z+fQ57777rthsNmlra+tVH7wIzWINjxrQu2ArVqwQu90uBw4ckPr6er2uXr2qz1m+fLm4XC7Zv3+/lJeXi9vtFrfbrY/33IafM2eOVFVVyd69e2XEiBG8Dc9i3YM1oAF0s41s3bpVn9Pa2iorV66U+Ph4iYqKkieffFLq6+vD1vP111/L3LlzJTIyUpKSkmTt2rXS0dHBAGKx7rG6XQBp/wmWYSUQCMBut6tug4huw+/3w2az3XScnwUjImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiU6dfvA6L7j6YBCxcCRiNQWQn4/cDFi0AopLqz23M4gPnzgS+/7K7GRqC5WXVXBDCAqJc0DViwABg3DujoAAIB4OuvgfPnuwPJ6+1+cre1AcGg6m7DjRgBLFsGiHT3/t//DdTXA6dOAefOddfFi0BrK9DVpbrb+wsDiPrEYACs1u4n9YgRwNSpwBNPdAdPYyNw5gzw5pvdgTSUaFr32ZvJBHznO901c2Z3KAUCwOXLwJ49wJ/+NDzO6u4VDCDqs54ziUAAaGgAqquB2lrgs8+6n8jX/D+UQ4ZId1292v0S7Isvuvs+caL7TK6+nuEz2BhA1GuXLztRUuJFeXl3yHz+efeZT2ur6s5uzWCIxcmTgsrKFtTUAN980x02bW3dQUrqMICoVzTNiMpKD95//33VrfSZ2TwGb7/dic8++0x1K3QN3oYnImUYQESkDAOIiJThNSCiO6BpGoxGI6T7f5gBgLCv6dYYQET9YDAYMHr0aPz4xz/GvHnz0NraisbGRjQ1NYVVIBCA3+8P+7O1tRUdHR3XVX9Dy2QywWQywWg0Xve11WpFdHQ0oqKiEB0dHVYRERE4cOAAqqurB/jo9KF3ZVsmGobMZjOysrLwk5/8BAsXLkRKSgoMhhtfyRARdHV1oaurC6FQSP+zra0NLS0tYdXc3Ay/36+HWGNjIxobGxEIBGC1WhEbG4uYmBjExsaGfR0TE4PIyMiblsVigcFguGFpmobz58/j1VdfxbZt29DS0jLIR5MBRNQrkZGRePTRR/Hss8/C4/HAbrdD07RbPkbTNP2M5NtsNhuSk5PDlt3JS7bb9XErLpcLb731FubPn4/169fj+PHjCA3iuzEZQES3kJCQAI/Hg+eeew7Tp09HZGTkHT3hb+ZurLO3zGYzPB4PHnnkEWzcuBEbN25EY2PjoGybAUR0DU3TkJqaih/96EfIz8/HhAkTYDabVbd1V2mahuTkZBQVFcHj8eCXv/wlDh8+jK67/OlcBhDRf5hMJowdOxZPP/00Fi9ejPT0dP1ayf3CZDJhxowZ2L17N7Zs2YJ33nkHtbW1d217fXof0ObNmzFx4kTYbDbYbDa43W589NFH+nhbWxsKCgqQmJiImJgYLFq0CL5rPplYW1uLvLw8REVFITk5GevWrUNnZ+fA7M0QZTAYMGLECOTm5qK4uBjr16/HD37wA4wbNw6xsbEwGo2qW7yvWa1WPProo9iyZQv279+Pn//85xg1ahSMRuN9FT49NE2D3W7HmjVrsHv3bixcuBAWi+WubKtPZ0BpaWl45ZVXkJGRARHB+++/jwULFqCyshITJkzAz372M/ztb3/Djh07YLfbsWrVKixcuBD//Oc/AQBdXV3Iy8uD0+nEJ598gvr6eixduhRmsxkvv/zyXdlBVSwWC9LS0jBjxgzMnj0bM2fORHp6un4qLyJoamqC1+vFmTNnUFVVhZMnT6K6uhoXLlxAIBAY1IuB9yO73Y7HH38cy5Ytw3e/+11ER0ffl4FzMwaDAZMmTcK2bdvwwQcfoLi4GF999dXAbkTuUHx8vPzhD3+QpqYmMZvNsmPHDn3szJkzAkDKyspERKSkpEQMBoN4vV59zubNm8Vms0kwGOz1Nv1+vwAYUqVpmsTExEh2drasXbtW9u7dKz6fTzo6OiQUCt12n0KhkASDQWlqapITJ07IX/7yFykqKpIf/vCHkpmZKbGxsWI2m5Xtn9FolPz8fOXHuT+VnZ0tkyZN0v+enE6n/PSnP5VPP/1U2traevX3c78LhUJSU1Mjy5Ytk6ioqF4fe7/ff8v19juAOjs75cMPPxSLxSKnT5+W0tJSASCNjY1h81wul7zxxhsiIrJ+/XqZNGlS2Pi5c+cEgBw/frzX2x4qAaRpmiQnJ8vs2bPl9ddfl/LycvH7/RIKhe74h7pnHZ2dnXLp0iWprq6WnTt3SlFRkSxYsEDGjBkjcXFxomkaA6gXATR58mR58MEHpaioSGpqaqSzs5PB0w+tra2yc+dOycrK6tXP3u0CqM8XoU+ePAm32422tjbExMRg165dyMzMRFVVFSwWC+Li4sLmOxwOeP/z6/G8Xi8cDsd14z1jNxMMBhH81u/5DAQCfW17wERERCA1NRUzZ87EnDlzMH36dP2l1UCevvesy2g0IiEhAQkJCRg/fjwWLlyIrq4uXLlyBefPn8e//vUvHD9+HCdOnEBNTQ0uXrwIg8GgvxPWaDTqde33vZnz7XfVJicnY+nSpQO2j4MlNjYW06dPx6xZs+BwOG76xkG6vYiICCxcuBA5OTnYvHkz3n33XVy+fLn/7+Lu6wPGjh2Lqqoq+P1+7Ny5E/n5+Th48GC/Nt5bxcXFeOmll+7qNm5G0zRERUVh3LhxeOyxx5Cbm4vJkycjISHhujeYDVY/JpMJdrsddrsdEyZMwIIFC9DR0YGWlhYEg0Fomtbv6rnrc6Pq2T7d3zRNQ1paGjZs2IBZs2Zh/fr1KC8vR3t7e5/X1ednkMViwYMPPggAyM7OxrFjx/Db3/4WixcvRnt7O5qamsLOgnw+H5xOJwDA6XTi6NGjYevruUvWM+dGCgsLsWbNGv37QCCA9PT0vrbea0ajEYmJicjOzsasWbPw+OOPIyMjA9HR0QCG5pPQbDYjPj5edRt0H7FYLHjsscewe/dubNy4EVu2bIHP5+vTe4fu+J/wUCiEYDCI7OxsmM1mlJaWYtGiRQCAmpoa1NbWwu12AwDcbjd+85vfoKGhQX8r+r59+2Cz2ZCZmXnTbVitVlit1jtt9To9Ly0SEhKQmpqKBx54AJmZmcjKysKUKVOQkpICq9U6JAOHaCjQNA1JSUn41a9+hby8PLz88sv4xz/+0evPlWnShxdvhYWFmDt3LlwuF5qbm/HBBx/g1Vdfxd///nfMnj0bK1asQElJCd577z3YbDa88MILAIBPPvkEQPdt+IcffhgpKSl47bXX4PV68fTTT+O5557r0234QCAAu93e6/kAEBUVhbi4ODgcDowdOxZjxozR/3Q6nYiPj0dUVJQ+n6FD1DcigubmZmzbtg1vvvkmzp07B7/fD5vNdtPH9OkMqKGhAUuXLkV9fT3sdjsmTpyohw8AvPnmmzAYDFi0aBGCwSA8Hg9+97vf6Y83Go3Ys2cPVqxYAbfbjejoaOTn5+PXv/51P3f5mp0xmWCxWGCz2eByufDAAw9g/PjxyMzMxKhRo5CamorExET9A4IMGaKBo2kabDYbVq5ciZycHEybNu32j+nLGdBQ0XMGNHLkSIwcORIZGRkYP348MjIykJGRgdTUVMTGxiI6Opp3PIgU6HmODugZ0FBTWlqK0aNHw2w233ef2SG6FwzrAEpNTUVERITqNoion/j6hIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGZPqBvpDRAAAgUBAcSdEdCM9z82e5+rNDMsAunTpEgAgPT1dcSdEdCvNzc2w2+03HR+WAZSQkAAAqK2tveXOUbhAIID09HTU1dXBZrOpbmdY4DHrHxFBc3MzUlJSbjlvWAaQwdB96cput/OHoh9sNhuPWx/xmPVdb04OeBGaiJRhABGRMsMygKxWKzZs2ACr1aq6lWGFx63veMzuLk1ud5+MiOguGZZnQER0b2AAEZEyDCAiUoYBRETKDMsA2rRpE0aNGoWIiAjk5OTg6NGjqltSpri4GFOnTkVsbCySk5PxxBNPoKamJmxOW1sbCgoKkJiYiJiYGCxatAg+ny9sTm1tLfLy8hAVFYXk5GSsW7cOnZ2dg7kryrzyyivQNA2rV6/Wl/GYDRIZZrZv3y4Wi0X++Mc/yunTp2XZsmUSFxcnPp9PdWtKeDwe2bp1q5w6dUqqqqpk3rx54nK5pKWlRZ+zfPlySU9Pl9LSUikvL5fp06fLjBkz9PHOzk7JysqS3NxcqayslJKSEklKSpLCwkIVuzSojh49KqNGjZKJEyfKiy++qC/nMRscwy6Apk2bJgUFBfr3XV1dkpKSIsXFxQq7GjoaGhoEgBw8eFBERJqamsRsNsuOHTv0OWfOnBEAUlZWJiIiJSUlYjAYxOv16nM2b94sNptNgsHg4O7AIGpubpaMjAzZt2+ffO9739MDiMds8Ayrl2Dt7e2oqKhAbm6uvsxgMCA3NxdlZWUKOxs6/H4/gP//gd2Kigp0dHSEHbNx48bB5XLpx6ysrAwPPfQQHA6HPsfj8SAQCOD06dOD2P3gKigoQF5eXtixAXjMBtOw+jDqxYsX0dXVFfaXDgAOhwNnz55V1NXQEQqFsHr1asycORNZWVkAAK/XC4vFgri4uLC5DocDXq9Xn3OjY9ozdi/avn07jh8/jmPHjl03xmM2eIZVANGtFRQU4NSpUzh8+LDqVoa0uro6vPjii9i3bx8iIiJUt3NfG1YvwZKSkmA0Gq+7G+Hz+eB0OhV1NTSsWrUKe/bswccff4y0tDR9udPpRHt7O5qamsLmf/uYOZ3OGx7TnrF7TUVFBRoaGvDII4/AZDLBZDLh4MGDePvtt2EymeBwOHjMBsmwCiCLxYLs7GyUlpbqy0KhEEpLS+F2uxV2po6IYNWqVdi1axf279+P0aNHh41nZ2fDbDaHHbOamhrU1tbqx8ztduPkyZNoaGjQ5+zbtw82mw2ZmZmDsyODaNasWTh58iSqqqr0mjJlCpYsWaJ/zWM2SFRfBe+r7du3i9Vqlffee0+qq6vl+eefl7i4uLC7EfeTFStWiN1ulwMHDkh9fb1eV69e1ecsX75cXC6X7N+/X8rLy8Xtdovb7dbHe24pz5kzR6qqqmTv3r0yYsSI++qW8rfvgonwmA2WYRdAIiLvvPOOuFwusVgsMm3aNPn0009Vt6QMgBvW1q1b9Tmtra2ycuVKiY+Pl6ioKHnyySelvr4+bD1ff/21zJ07VyIjIyUpKUnWrl0rHR0dg7w36lwbQDxmg4O/joOIlBlW14CI6N7CACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJT5f0afP9HrRrMVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box([-1.5       -1.5       -5.        -5.        -3.1415927 -5.\n",
      " -0.        -0.       ], [1.5       1.5       5.        5.        3.1415927 5.        1.\n",
      " 1.       ], (8,), float32)\n",
      "env.action_space= Box(-1.0, 1.0, (2,), float32)\n",
      "state= [ 0.00262108  1.416303    0.2654778   0.23924235 -0.00303044 -0.06013473\n",
      "  0.          0.        ]\n",
      "action= [-0.6357824  -0.06871741]\n",
      "next_state= [ 0.00524225  1.4211081   0.265119    0.21354581 -0.0060019  -0.05943376\n",
      "  0.          0.        ]\n",
      "reward= 0.9163025709748922\n",
      "done= False\n",
      "info= {}\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x7f3f9857f430>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = PPO(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    n_steps=1024,  #运行N步后执行更新,buffer_size=n_steps*环境数量\n",
    "    batch_size=64,  #采样数据量\n",
    "    n_epochs=16,  #每次采样后训练的次数\n",
    "    gamma=0.99,\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-270.40662152247387, 157.04002772963742)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a137a618d1904da0a03c0b9c211ce992",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# #训练\n",
    "# model.learn(100_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/2.PPO.Lunar Lander Continuous')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(251.44117638377293, 64.62917913218226)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = PPO.load('save/2.PPO.Lunar Lander Continuous')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "414e1478",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAbnUlEQVR4nO3de3BU5f0/8PfZSxZCspvLkl1iEhKUCsilNhBYnF9xNBIBEZXpiBMgYqoDBiti7ZBp1dppG2s72moVnQGj9VIKItVBxGK4WMsKJCQIIcQLDYmY3TSE7AZINpf9fP+I2Z+rgVxI9snC+zXzzJBznuz5nMPue59znj0bTUQEREQK6FQXQESXLwYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpoyyAnn/+eaSmpmLYsGGYPn069u/fr6oUIlJESQD94x//wOrVq/H444/j4MGDmDJlCrKyslBXV6eiHCJSRFNxM+r06dMxbdo0/PWvfwUA+P1+JCcn44EHHsCaNWtCXQ4RKWII9QZbW1tRUlKC/Pz8wDKdTofMzEw4nc5uf8fn88Hn8wV+9vv9aGhoQHx8PDRNG/SaiahvRARNTU1ITEyETnf+E62QB1B9fT06Ojpgs9mClttsNhw7dqzb3ykoKMATTzwRivKIaADV1NQgKSnpvOvDYhYsPz8fHo8n0Kqrq1WXRES9EB0dfcH1IR8BWa1W6PV6uN3uoOVutxt2u73b3zGZTDCZTKEoj4gGUE+XSEI+AoqIiEB6ejqKiooCy/x+P4qKiuBwOEJdDhEpFPIREACsXr0aOTk5mDp1KjIyMvDnP/8ZZ8+exbJly1SUQ0SKKAmgO++8E//73//w2GOPweVy4Yc//CG2b9/+vQvTRHRpU/I5oIvl9XphsVhUl0FEPfB4PDCbzeddHxazYER0aWIAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETK9DmAPvroI8yfPx+JiYnQNA3//Oc/g9aLCB577DGMGjUKw4cPR2ZmJj7//POgPg0NDcjOzobZbEZMTAxyc3Nx5syZi9oRIgo/fQ6gs2fPYsqUKXj++ee7Xf/UU0/h2WefxYsvvoh9+/ZhxIgRyMrKQktLS6BPdnY2ysvLsWPHDmzduhUfffQR7rvvvv7vBRGFJ7kIAGTLli2Bn/1+v9jtdvnjH/8YWNbY2Cgmk0n+/ve/i4jI0aNHBYAcOHAg0Of9998XTdPk5MmTvdqux+MRAGxsbEO8eTyeC76WB/Qa0H//+1+4XC5kZmYGllksFkyfPh1OpxMA4HQ6ERMTg6lTpwb6ZGZmQqfTYd++fd0+rs/ng9frDWpEFP4GNIBcLhcAwGazBS232WyBdS6XCwkJCUHrDQYD4uLiAn2+q6CgABaLJdCSk5MHsmwiUiQsZsHy8/Ph8XgCraamRnVJRDQABjSA7HY7AMDtdgctd7vdgXV2ux11dXVB69vb29HQ0BDo810mkwlmszmoEVH4G9AASktLg91uR1FRUWCZ1+vFvn374HA4AAAOhwONjY0oKSkJ9Nm5cyf8fj+mT58+kOUQ0VDXh0kvERFpamqS0tJSKS0tFQDy9NNPS2lpqZw4cUJERJ588kmJiYmRd955Rz799FNZsGCBpKWlSXNzc+Axbr75Zrn22mtl37598vHHH8vYsWPlrrvu6nUNnAVjYwuP1tMsWJ8DaNeuXd1uKCcnR0Q6p+IfffRRsdlsYjKZ5MYbb5TKysqgxzh16pTcddddEhUVJWazWZYtWyZNTU0MIDa2S6z1FECaiAjCjNfrhcViUV0GEfXA4/Fc8JptWMyCEdGliQFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnTpwAqKCjAtGnTEB0djYSEBNx2222orKwM6tPS0oK8vDzEx8cjKioKCxcuhNvtDupTXV2NefPmITIyEgkJCXjkkUfQ3t5+8XtDROFF+iArK0sKCwvlyJEjUlZWJnPnzpWUlBQ5c+ZMoM/y5cslOTlZioqKpLi4WGbMmCEzZ84MrG9vb5eJEydKZmamlJaWyrZt28RqtUp+fn6v6/B4PAKAjY1tiDePx3PB13KfAui76urqBIDs2bNHREQaGxvFaDTKpk2bAn0qKioEgDidThER2bZtm+h0OnG5XIE+a9euFbPZLD6fr1fbZQCxsYVH6ymALuoakMfjAQDExcUBAEpKStDW1obMzMxAn3HjxiElJQVOpxMA4HQ6MWnSJNhstkCfrKwseL1elJeXd7sdn88Hr9cb1Igo/PU7gPx+P1atWoXrrrsOEydOBAC4XC5EREQgJiYmqK/NZoPL5Qr0+Xb4dK3vWtedgoICWCyWQEtOTu5v2UQ0hPQ7gPLy8nDkyBFs2LBhIOvpVn5+PjweT6DV1NQM+jaJaPAZ+vNLK1euxNatW/HRRx8hKSkpsNxut6O1tRWNjY1BoyC32w273R7os3///qDH65ol6+rzXSaTCSaTqT+lEtFQ1peLzn6/X/Ly8iQxMVE+++yz763vugj91ltvBZYdO3ZMgO9fhHa73YE+L730kpjNZmlpaelVHbwIzcYWHm1AZ8FWrFghFotFdu/eLbW1tYF27ty5QJ/ly5dLSkqK7Ny5U4qLi8XhcIjD4Qis75qGnz17tpSVlcn27dtl5MiRnIZnY7sE24AG0Pk2UlhYGOjT3Nws999/v8TGxkpkZKTcfvvtUltbG/Q4VVVVMmfOHBk+fLhYrVZ5+OGHpa2tjQHExnaJtZ4CSPsmWMKK1+uFxWJRXQYR9cDj8cBsNp93Pe8FIyJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhAREOUpmmwWq0wGo2qSxk0DCCiIUan02HMmDH43e9+B6fTiQ0bNuCOO+5AfHw8NE1TXd6A4r1g1CuaBtxxB6DXA6WlgMcD1NcDfr/qynpmswHz5wNfftnZTp8GmppUV9W9tLQ03HvvvViyZAmuuOIKaJoGEUF7ezsqKyuxefNmbN68GRUVFWHxl2R6uheMAfQtRqMRiYmJ+MEPfoCSkhI0NDQM+DbClU4HvPIKMG4c0NYGeL1AVRXw1VedgeRydb64W1oAn091tcEmTgTWrwdEOmv/+mugthY4cgQ4fryz1dcDzc1AR0fo69PpdEhLS0Nubi6WLl2KUaNGQafr/uTE7/ejsbERe/bswZtvvomdO3fi9OnTGKovYwZQD/R6PZKSkjBr1izMnz8f1113HWJjY3Hs2DGsX78emzdvRm1t7YBsK5x1BdCECd9fJ9IZPKdPAxUVwDPPdAbSUDFxIvDyy5378G0inc3rBRoagK1bgddfD92oTtM0pKWl4ac//SmWLl2KxMTEPp1itba24vPPP8fmzZuxadMmVFZWoq2tbRAr7jsGUDe6RjrXX389FixYgBkzZiAhIQE6nS7wBBAR+P1+fPnll3jttdfwxhtvoKqqasi+0wy2bwdQ10jC6wXq6oCjR4HqauDQoc4XssvV2Weo6AogTeus69y5zlOwzz/vrPvTTztHcrW1oTk10zQNY8aMCQTPqFGjLurajogEjYp27dqF+vr6Aay4/xhA3zAYDEhOTsaPf/xj3HrrrZg5cyasVisMhp6/Ftvv9+PkyZPYuHEjCgsLcezYMXSoGKsrpNMBTz9th9frQnEx4HYDn33WOfJpblZd3YVNnhyNBx8UlJaeQWUlcOJEZ9i0tHQGaah0Bc+9996LxYsXX/BUq7+6RkVbtmzBpk2bUFFRoXRU1FMAXdQfJlSlt9+IaDAYZPTo0bJs2TJ5++23pba2Vjo6OsTv9/dru36/X+rq6mTdunWSkZEhRqNR+TfOharp9XrJyclRXkd/Wnp6ukyZMkXZ9jVNkyuvvFL+8Ic/yMmTJ/v9/Ovrc7WhoUHeeecd+clPfiLx8fFK9r2nb0Ts11/FGMqMRmPgmk7X6VVvRzo90TQNI0eOxD333IOFCxfiww8/xIsvvoi9e/eieagPA0IkNSUDVyb/GAACp6siAul6Tnb9WwSNZ77C4aPvwi+X5miyazq9a8Rjt9sHfMRzPpqmITY2FvPnz8ecOXOCRkXHjh2Db4jMFFwSAaTX65GSkoLrr78e8+fPh8PhCFzTGQyapiEmJgYLFy7E3Llz8fHHH+PFF1/Ehx9+iKahOr8bAjqdEdOuycH4pPnQoEfnm2D3BIL6M8dwoqYYjd5L6++8fftUa8mSJRd9jediazEajZgwYQLGjx+PFStW4N///jfeeOONIXGtKKwDaPTo0bjxxhuxYMECTJ8+HVarNehC8mDTNA2RkZG46aabMGvWLJSUlGD9+vV49913Q/Ifq2kahg0bhsjISCQmJiI1NRV1dXU4ePCgkvP+EcPjERVpgwb9N/8H5/9/0ACYDGZAC4MPEvWSTqfDlVdeGTTiGUofHNQ0DXFxcbj11lsxd+5cfPnll3A6ndi7dy9KS0tx/PhxNDU1hfTzRWEdQB9++CHS0tKg1+uV1qFpGkwmE2bOnIlp06bhZz/7GQoLC7Fx40a4XK6LnjnT6XTQ6XSIjY2FzWZDSkoKxo8fj/Hjx2PMmDFIS0tDTEwMIiMjce7cObz77rt49tlnUVZWFtKL5THmJMSMSOr1i86oj0Tk8Dg0ek4OcmWDS9O0oOBROeLpja5R0bhx4zBu3Djk5OTgzJkz+Prrr3Ho0CF88sknOHDgAL744gvU19cP6nMorGfBerzCroh0/rkjVFVV4bXXXsPrr7+O48ePw9+LD5gMHz4cUVFRSEhIwFVXXYWxY8cGniijRo2C1WpFVFRU4Ane3RNdvpmWffvtt/Hcc8/hyJEjF/0k0uv1WLx4MV599dXz9rk+4yFkXLMMkUZrrx6z3d+C9z5Zg9IjGy+qtp6kp6ejvb0dhw4dGtDH7XoR5+TkIDs7GzabbUgHT290xUFraytqa2tRUVEBp9OJAwcOoLy8HPX19X263tnTazSsR0BDlaZpgesAjz76KHJzc/HWW2+hsLAQR48ehd/vh8FggNlsRmJiIpKTkzF+/HhMmDABaWlpGD16NKxWK0wmE4xGY5+f1F0XIO+55x7Mnz8fGzduxNq1a1FZWTlo72Z6XQQs0Ykw6Ib1/ne0CERFjkTnCVn4vA+OGDECDocD99xzD2666aZL6h6trv0wmUxITU1Famoqbr75Zvh8Ppw+fRqVlZUoLi7G3r17UV5ejpqaGvh8vl69uXa7PY6AQkNE0NDQgO3bt6OlpQVXX301kpOTERMTg+jo6AuOaAZi2263G2+88QZeeuklfPHFF30+LexpBBQdacOiW9ZhlHkKdLreva+JCA5V/QPvFv0Cfv/gXXcYqBFQQkICbrnlFtx9992YNm0aTCbTJRM8vdX1vPH7/Th9+jSqqqpQUlKCAwcOoLi4GCdOnIDH4wn04whoiNA0DfHx8cjOzlaybbvdjoceegiLFi3Cq6++inXr1g3IJ7ujoxPg93dA0/TwSzs0rW/X46IjE6DTGWAwmDBsmBle79C67cVgMGDs2LFYtGgRFi1ahDFjxkCv1192wdOla7/1ej2sViusVivS09ORm5uL5uZmVFVV4fDhw9i1axfWrVvX4+MxgC4jOp0OV1xxBdasWYPs7GysX78ehYWF+Oqrr/r1eFHDR2LO/3sCRsMIFB/9G7QLzHqdj0AQHzsaszIehPgF7xT9Aq2tZ/tVz0AaPnw4MjIysGzZMsyZMwcjR468bEOnJ5qmwWAwIDo6GpMmTcKkSZNwyy239CqA+H1AlyGdTofRo0fj17/+Nfbs2YM1a9YgMTGxz4+TYB2HUfFTEGmKRWNjP2eyRNDiO4PIiHikjJwBa9yV/XucATJy5EgsXrwYW7duxXvvvYelS5ciISGB4TNI+hRAa9euxeTJk2E2m2E2m+FwOPD+++8H1re0tCAvLw/x8fGIiorCwoUL4Xa7gx6juroa8+bNQ2RkJBISEvDII4+ExfeaXIq6Pqn729/+FkVFRVi1ahUSEhJ69buapkPKFVMRaYzD8a//jXPNp9EhrTjbVoezrd9t//tOq8e5tno0t59Cq/8czjU3oMZdjAhDFMYkz8SFPj80GPR6Pa666ir88pe/xK5du/Dyyy/jhhtuwIgRIxg8g6xPp2BJSUl48sknMXbsWIgIXn31VSxYsAClpaW45ppr8NBDD+G9997Dpk2bYLFYsHLlStxxxx34z3/+AwDo6OjAvHnzYLfbsXfvXtTW1mLp0qUwGo34/e9/Pyg7SD3T6/W4+uqr8ac//Qm5ubl44YUXsHHjRpw6deq8vzMswozRo6aj3e/DFyd2w9d6BhWf74Bet/Nb81nfvb703TWCU43H0d7eghMn92PSlQtxVdINOPDpG/D5Bv8T5cOGDUN6ejruvvtu3HLLLbDZbAAGZyKAzuNib3qLjY2VdevWSWNjoxiNRtm0aVNgXUVFhQAQp9MpIiLbtm0TnU4nLpcr0Gft2rViNpvF5/P1eptdN6P2dKMb9U9bW5scPHhQcnNzJTY2ttubUVOTHLI6p1juvn2jDDOZL/qmxajIBFl625vyi7sPS/IV6YN6M2pcXJzceeedsn37dvF6vSG5OfRy09vXaL+vAXV0dGDDhg04e/YsHA4HSkpK0NbWhszMzECfcePGISUlBU6nEwDgdDoxadKkwDsNAGRlZcHr9aK8vLy/pdAAMxgMuPbaa7F27Vps374d2dnZiI6ODuqTmjQdIu344qudaBmA0cqZc//D13WH0CE+pCZlQNMG9vJk17cO/vznP8fOnTvxt7/9DbNnzw76CASFXp9nwQ4fPgyHw4GWlhZERUVhy5YtmDBhAsrKyhAREYGYmJig/jabDa5vvh7P5XIFhU/X+q515+Pz+YLu3vV6vX0tm/rBaDRi2rRpKCwsRHFxMf71r39h6dKlAAB7/ChcMaoRWtwojBm/ZEC2l5hgRYL1JNItI5F29eIBeUwAiI6ORkZGBm644QYkJiaG7I506lmfA+jqq69GWVkZPB4P3nrrLeTk5GDPnj2DUVtAQUEBnnjiiUHdBnWv674hh8MBh8PRTY+sQdjqrYPwmDQU9fmtICIiAldddRXS09NRUFCAKVOm4C9/+QvsdjtaW1vR2NgY1N/tdsNutwMA7Hb792bFun7u6tOd/Px8eDyeQKupubS+voHocnXRY1G/3w+fz4f09HQYjUYUFRUF1lVWVqK6ujrwzulwOHD48GHU1dUF+uzYsQNmsxkTuvu282+YTKbA1H9XI6Lw16dTsPz8fMyZMwcpKSloamrCm2++id27d+ODDz6AxWJBbm4uVq9ejbi4OJjNZjzwwANwOByYMWMGAGD27NmYMGEClixZgqeeegoulwu/+tWvkJeXB5PJNCg7SERDV58CqK6uDkuXLkVtbS0sFgsmT56MDz74ADfddBMA4JlnnoFOp8PChQvh8/mQlZWFF154IfD7er0eW7duxYoVK+BwODBixAjk5OTgN7/5zcDuFRGFBd4NT0QDrrevUc5HEpEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkjEF1Af0hIgAAr9eruBIi6k7Xa7PrtXo+YRlAp06dAgAkJycrroSILqSpqQkWi+W868MygOLi4gAA1dXVF9w5Cub1epGcnIyamhqYzWbV5YQFHrP+ERE0NTUhMTHxgv3CMoB0us5LVxaLhU+KfjCbzTxufcRj1ne9GRzwIjQRKcMAIiJlwjKATCYTHn/8cZhMJtWlhBUet77jMRtcmvQ0T0ZENEjCcgRERJcGBhARKcMAIiJlGEBEpExYBtDzzz+P1NRUDBs2DNOnT8f+/ftVl6RMQUEBpk2bhujoaCQkJOC2225DZWVlUJ+Wlhbk5eUhPj4eUVFRWLhwIdxud1Cf6upqzJs3D5GRkUhISMAjjzyC9vb2UO6KMk8++SQ0TcOqVasCy3jMQkTCzIYNGyQiIkJefvllKS8vl3vvvVdiYmLE7XarLk2JrKwsKSwslCNHjkhZWZnMnTtXUlJS5MyZM4E+y5cvl+TkZCkqKpLi4mKZMWOGzJw5M7C+vb1dJk6cKJmZmVJaWirbtm0Tq9Uq+fn5KnYppPbv3y+pqakyefJkefDBBwPLecxCI+wCKCMjQ/Ly8gI/d3R0SGJiohQUFCisauioq6sTALJnzx4REWlsbBSj0SibNm0K9KmoqBAA4nQ6RURk27ZtotPpxOVyBfqsXbtWzGaz+Hy+0O5ACDU1NcnYsWNlx44dMmvWrEAA8ZiFTlidgrW2tqKkpASZmZmBZTqdDpmZmXA6nQorGzo8Hg+A/3/DbklJCdra2oKO2bhx45CSkhI4Zk6nE5MmTYLNZgv0ycrKgtfrRXl5eQirD628vDzMmzcv6NgAPGahFFY3o9bX16OjoyPoPx0AbDYbjh07pqiqocPv92PVqlW47rrrMHHiRACAy+VCREQEYmJigvrabDa4XK5An+6Oade6S9GGDRtw8OBBHDhw4HvreMxCJ6wCiC4sLy8PR44cwccff6y6lCGtpqYGDz74IHbs2IFhw4apLueyFlanYFarFXq9/nuzEW63G3a7XVFVQ8PKlSuxdetW7Nq1C0lJSYHldrsdra2taGxsDOr/7WNmt9u7PaZd6y41JSUlqKurw49+9CMYDAYYDAbs2bMHzz77LAwGA2w2G49ZiIRVAEVERCA9PR1FRUWBZX6/H0VFRXA4HAorU0dEsHLlSmzZsgU7d+5EWlpa0Pr09HQYjcagY1ZZWYnq6urAMXM4HDh8+DDq6uoCfXbs2AGz2YwJEyaEZkdC6MYbb8Thw4dRVlYWaFOnTkV2dnbg3zxmIaL6KnhfbdiwQUwmk7zyyity9OhRue+++yQmJiZoNuJysmLFCrFYLLJ7926pra0NtHPnzgX6LF++XFJSUmTnzp1SXFwsDodDHA5HYH3XlPLs2bOlrKxMtm/fLiNHjrysppS/PQsmwmMWKmEXQCIizz33nKSkpEhERIRkZGTIJ598orokZQB02woLCwN9mpub5f7775fY2FiJjIyU22+/XWpra4Mep6qqSubMmSPDhw8Xq9UqDz/8sLS1tYV4b9T5bgDxmIUGv46DiJQJq2tARHRpYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETK/B8HhSlm+/rMKAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "266.9408517815397 171 [1.8299488652012883, 1.9649769051223984, 2.010039139768139, 2.0438045380105336, 2.066386786754265, 2.070322877108822, 2.041754027990862, 1.953044427117561, 1.7477683884659143, 1.3195227671954228, 0.550252749318247, -0.45065189390112437, -1.285118543664396, -1.7716930396991586, -2.0096940424465686, -2.1150649757676945, -2.152399718541716, -2.1533388896427823, -2.1335824361738673, -2.1013945397513965, -2.0613617685707766, -2.01612723635796, -1.9673619849544934, -1.916079989134147, -1.9543691028086414, -1.8063912271767038, -1.750898103582358, -1.6946710247575538, -1.6378091198812967, -1.58048418024336, -1.5227725428478323, -1.464741892070208, -1.4064540485665873, -1.3479727307947655, -1.289310538155405, -1.230486911881826, -1.1715421892810127, -1.1125101060586076, -1.05339636742508, -0.830260874718349, -0.8252553029220167, -0.7658465308285827, -0.706386894426231, -0.6468898411372379, -0.5873880159856242, -0.5278389684597471, -0.4682954487392408, -0.4087408357902689, -0.34918176191038697, -0.28960795456342225, -0.23004776188039955, -0.17050160250835233, 3.274240991801122, 1.2227239362171816, 1.8929920254333807, 2.513542564417918, 2.9487706094546295, 2.9893667360220944, 3.435579065824231, 3.1512450953859457, 5.383896369556095, 3.401660163998298, 5.076774581479663, 2.2152946322271303, 1.6652183707321797, 5.6346090366991515, 3.7964190405816054, 5.308212657199346, 3.358916413968342, 2.556457265015001, 1.3100802674566239, 3.9432190099603077, 5.202042666767352, 1.3760966187209192, 2.102759119944784, 2.818533525335721, 4.587411141481303, 3.4335014682234486, 1.726565245846075, 2.6103222144439058, 2.478950047427406, 4.994046232197829, 0.6902524937738321, 1.9403848229207654, 1.3025542651294217, 3.03037309547116, 2.678057738993272, 3.2763627436461435, 4.749884500989038, 0.6040248353856612, 3.0730474880199323, 3.7511609941394624, 1.846815124113553, 1.5880603212780187, 2.8394819415429056, 3.439340935869231, 0.3815829172918626, 1.7882949279575626, 3.7287940195668, 0.014631844654896031, 1.5015108049656802, 0.09836702285903698, -0.04037840897741346, 1.7146824976734976, 0.4403562245148578, 4.041992764219811, 2.682340066532203, -0.3783384511956967, -0.9038996254018741, 0.9899090008074296, 3.6166860246655945, 24.756628493292634, 3.270358514234676, 4.182609584244207, 3.515107059762127, 2.885383811872699, 1.3635083528402538, -5.65505100677504, -0.31790552910002745, 0.27846785862103474, 2.3468459229453664, 1.9289849532770553, 1.7485034837599365, 1.1829069702141464, 0.6638092546410874, 0.5487156245500557, 0.42888298328381147, -0.4045112761659606, -0.41712761267933907, -0.22106603496883714, 0.049176596853177656, 0.3185012468129518, 0.5234895318763026, 0.630319205538683, 0.6362076438749185, 0.5631488559571576, 0.44697520504606914, 0.32551847363459174, 0.2290223900778532, 0.17471991677381915, 0.1656519467512485, 0.19332271635964027, 0.24232484028250134, 0.29573588701974884, 0.3395094683589104, 0.36521153578020993, 1.7865641392452716, 0.012690639664969083, 0.02386910455246216, -0.0013619481594595584, 0.0005878741643350338, 0.0035141434762380186, 0.0007534600399594638, -0.00023710499543483365, 0.0005321640392672577, 0.0003226016030843937, -0.00010308302506700784, 7.296889665298067e-05, 8.153006074529401e-05, -1.924863900981677e-05, 5.409295519953616e-06, 1.1485167050295786e-05, 2.432366724036683e-06, -2.92220327224868e-06, 2.6047512804439066e-06, 8.303370471196558e-07, 2.2072652043902963e-06, 1.3497313489096996e-07, -2.470603119419934e-06, -2.6188535162674498e-06, 100]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 5 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
